#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2024/2/20 17:58
# @Author  : 王凯
# @File    : runtime_settings.py
# @Project : scrapy_spider
from apps.tax_policy.tax_policy.settings import *  # 导入原有配置

# ###################################### 部署时的配置 #################################################################
from components.settings.private.wfq_dev_redis_settings import *  # 导入 REDIS_URL

# 布隆过滤器
SCHEDULER = "components.component.scrapy_redis_bloomfilter_custom.scheduler.Scheduler"

# Ensure all spiders share same duplicates filter through redis.
DUPEFILTER_CLASS = "components.component.scrapy_redis_bloomfilter_custom.dupefilter.RFPDupeFilter"

# Number of Hash Functions to use, defaults to 6
BLOOMFILTER_HASH_NUMBER = 6

# Bit
BLOOMFILTER_BIT = 10

DONT_FILTER_REQUEST_PARSE_NAME_LIST = [
    "parse_list",
]  # 不过滤请求的解析函数名称

# 不清空数据
SCHEDULER_PERSIST = True

# 请求队列的生命周期 TTL 不配置默认为86400*7,即7天
SCHEDULER_QUEUE_KEY_EXPIRE_TIME = 86400 * 7

SPIDER_MIDDLEWARES = {
    # 不过滤请求的中间件
    "components.middlewares.spidermiddlewares.public.dont_filter_request_parse_name.DontFilterRequestParseNameSpiderMiddleware": 300,
}
